/***************************************************************************
                          tokenizer.cpp  -  description
                             -------------------
    begin                : Thu Mar 14 2002
    copyright            : (C) 2002 by Harry Kalogirou
    email                : harkal@gmx.net
 ***************************************************************************/

#include "stdafx.h"

#include "tokenizer.h"

#ifdef MEMPROF
#include "mmgr.h"
#endif

CTokenizerBase::CTokenizerBase(const string & b)
{
    lowerCaseTokens = false;
    line = 1;
    pos = 0;
    token = "";
    endOfStream = false;
    buffer = b;
}

string CTokenizerBase::lowerCase(string & a)
{
    string s = a;
    for (int i = 0; i < (int)s.length(); i++) {
        s[i] = s[i] <= 'Z' && s[i] >= 'A' ? s[i] + ('a' - 'A') : s[i];
    }
    return s;
}

/*bool CTokenizerBase::goBack()
{
  int size = 0;

  if( poss == 0 )
    return false;

  string newToken("");

  if(currentChar() == '\"'){
		while(1){
			if( !previousChar() )
        break;
      ++size;

			if(endOfStream){
				//Exception to handle open quotation
				break;
			}
			if(currentChar() == '\"'){
        continue;
  		}
			if(currentChar() == '\n')
				newToken += " ";
			else
				newToken += currentChar();
		}
	}
	do {
		newToken += currentChar();
		if( !previousChar() )
      break;
    ++size;

		if(endOfStream)break;
	} while (currentChar() > 32);
	
  for( int i=0; i<size; i++ )
    token[i] = newToken[size-i-1];

	return true;

}

bool CTokenizerBase::previousChar()
{
  if( pos == 0 )
    return false;

  if( buffer[pos] == '\n')
      line--;
  pos--;
  return true;
} */



void CTokenizerBase::nextChar()
{
    pos++;
    if (pos >= (int)buffer.length()) {
        endOfStream = true;
        pos--;
        return;
    }
    switch (buffer[pos]) {
    case 0:
        endOfStream = true;
        break;
    case '\n':
        line++;
        break;
    case '\r':
        buffer[pos] = ' ';
        break;
    }
}

void CTokenizerBase::expect(const string & token)
{
    nextToken();
    /*if (getToken() != token)
        throw CTokenizerBase::CTokenizerException(*this,
                                                  "Expected '" + token + "'");*/
}

void CTokenizerBase::skipToNextLine()
{
    while (currentChar() != '\n') {
        nextChar();
        if (endOfStream)
            return;
    }
    nextChar();
}

void CTokenizerBase::setLowerCaseTokens(bool b)
{
    lowerCaseTokens = b;
}

void CTokenizerBase::skipToToken(const string & token)
{
    while (nextToken()) {
        if (getToken() == token)
            return;
    }
}

void CTokenizerBase::backToStartOfLine()
{
    int i = (int)buffer.rfind("\n", pos);
    if (i == string::npos)
        pos = 0;
    else
        pos = i + 1;
}

void CTokenizerBase::skipWhitespace()
{
    while (currentChar() <= ' ') {
        nextChar();
        if (endOfStream)
            return;
    }
}

bool CTokenizerBase::nextToken()
{
    string newToken("");

    skipWhitespace();
    if (endOfStream)
        return false;

    // Handle quoted strings
    if (currentChar() == '\"') {
        while (1) {
            nextChar();
            if (endOfStream) {
                //Exception to handle open quotation
                token = newToken;
                return true;
            }
            if (currentChar() == '\"') {
                nextChar();
                token = newToken;
                return true;
            }
            if (currentChar() == '\n')
                newToken += " ";
            else
                newToken += currentChar();
        }
    }

    do {
        newToken += currentChar();
        nextChar();
        if (endOfStream)
            break;
    } while (currentChar() > 32);

    token = newToken;
    return true;
}

string & CTokenizerBase::getToken()
{
    if (lowerCaseTokens)
        token = lowerCase(token);

    return token;
}

int CTokenizerBase::getLine()
{
    return line;
}

//
// CTokenizer
//

bool CTokenizer::nextToken()
{
    int lastpos = pos;

    if (!CTokenizerBase::nextToken())
        return false;

    vector < CSeparator >::iterator iter;
    int min = (int)token.length();
    for (iter = separators.begin(); iter != separators.end(); iter++) {
        int i = (int)token.find(iter->sep);
        if (i == string::npos)
            continue;
        else if (i == 0) {
            pos -= (int)token.length() - (int)iter->sep.length();
            token = iter->sep;
            return true;
        } else if (i < min) {
            min = i;
        }
    }
    pos -= (int)token.length() - min;
    token.assign(token, 0, min);

    return true;
}

void CTokenizer::addSeparator(const CSeparator & s)
{
    separators.push_back(s);
}

void CTokenizer::addSeparator(const string & s)
{
    separators.push_back(CSeparator(s));
}

//
// CTokenizerNoComments
//

bool CTokenizerNoComments::nextToken()
{
    while (CTokenizer::nextToken()) {
        if (token == "//") {
            skipToNextLine();
        } else {
            return true;
        }
    }
    return false;
}
